

from xml.dom.ext.reader import HtmlLib

Reader = HtmlLib.Reader()

import re
p = re.compile(r"(<blockquote>|<div.*>)")

s1=open("xpa.html").read()
s=p.sub("", s1)

doc = Reader.fromString(s)

from xml.dom.ext import Print

def get_h4_siblings(node):
    expl, syntax, example = [], [], []

    node_list = expl
    
    n_node = node.nextSibling
    while n_node and (n_node.nodeName != u"H4"):
        tl = collect_text(n_node)
        #print tl
        for tl1 in tl:
            if tl1.strip().startswith(u"Syntax:"):
                node_list = syntax
                break
        #elif tl and tl[0].startswith(u"Example:"):
        #    node_list = example
            
        node_list.append(n_node)
        n_node = n_node.nextSibling

    return expl, syntax #, example



def expl_to_string(expl):

    expl_string_ = []
    [expl_string_.extend(collect_text(f)) for f in expl]
    expl_string = " ".join(expl_string_).strip()
    return expl_string

from subprocess import Popen, PIPE

#import ConfigParser
#par2 = ConfigParser.RawConfigParser()

def syntax_to_string(syntax):

    P = Popen("w3m -T text/html -dump",
              shell=True, stdin=PIPE, stdout=PIPE)
    pipe = P.stdin

    for s in syntax:
        Print(s, stream=pipe)

    return P.communicate()[0]


def make_readable(ss):
    ss = ss.replace(u"\xa0 "," ")
    ss = ss.replace(u"\xa0"," ")
    ss = ss.replace("&lt;","<")
    ss = ss.replace("&gt;",">")
    ss = ss.replace("\n","")
    ss = ss.strip()
    
    return ss

def collect_text(node):
    "A function which collects text inside 'node', returning that text."

    s = []
    if node.nodeType == node.TEXT_NODE:
        ss = make_readable(node.nodeValue)
        s.append(ss)
    elif node.nodeName == u"BR":
        s.append(u"\n")
        #print "<BR>"
    
    for child_node in node.childNodes:
        s.extend(collect_text(child_node))

    return s

help_dict = {}
help_dict_ = {}
hhh = {}

h4_elements = doc.getElementsByTagName("h4")
for h4 in h4_elements:
    a_elements = h4.getElementsByTagName("a")
    if a_elements:
        nn = a_elements[0].getAttributeNS(None, "name")
        #print a_elements[0].getAttributeNS(None, "name")

        hhh[nn] = h4
        aaa = get_h4_siblings(h4)
        #print aaa[0].encode("ascii", "replace")
        #print

        expl = expl_to_string(aaa[0])
        syntax = syntax_to_string(aaa[1])
        help_dict_[nn] = aaa
        help_dict[nn] = dict(short_expl=expl.split(".")[0],
                             expl=expl,
                             syntax=syntax)
        
        
import pickle
pickle.dump(help_dict, open("ds9_xpa_help.pickle","w"))

if 0:
    hhh=open("hhh.txt", "w")
    for k in help_dict.keys():
        #print k + ":" + help_dict[k]["short_expl"]
        hhh.write('[%s]\n' % k)
        hhh.write(help_dict[k]["short_expl"])
        hhh.write("\n")
        hhh.write(help_dict[k]["expl"])
        hhh.write("\n")
        hhh.write(help_dict[k]["syntax"])
        hhh.write("\n")
        hhh.write("\n")

    hhh.close()
